import time
from lxml import etree
import pymysql
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from user_agent import ua_list  # 导入用户代理池
from random import choice

# 获取 url 列表
def opearte_url():
    # 由于时间原因，这里只获取两个主页，每个主页获取两个子页
    urls = []
    for i in range(2):
        for j in range(2):
            url = f'http://www.wuqibaike.com/index.php?category-view-{i+1}-{j+1}'
            urls.append(url)
    return urls

# 获取词条信息
def get_info():
    ### 利用selenium加载网页（调用浏览器） ###
    driver = webdriver.Chrome()
    # 调用 opearte_url
    urls = opearte_url()
    print(urls)
    # 创建空列表来保存词条摘要
    content_list = []
    for url in urls:

        # 设置主页超时时间
        driver.set_page_load_timeout(20)
        try:
            # 发送请求
            driver.get(url)
            time.sleep(15)
        except TimeoutException:
            print('页面超时')
        # 滚动页面，可以不设置
        for i in range(10):
            js = f"document.documentElement.scrollTop={1000 * (i + 1)}"
            driver.execute_script(js)
            time.sleep(3)

        ### 使用 Xpath 获取词条和词条摘要内容  #####
        #获取html源码
        html = driver.page_source
        # print(html)
        html_data= etree.HTML(html)
        # 获取词条
        key_words = html_data.xpath('//a[@class="clink f20"]/text()')
        # 获取词条对应的摘要
        contents = html_data.xpath('//*[@id="html"]/body/section/div//p/text()')
        # 去掉摘要中的 “摘要：”，并存进列表content_list中
        for content in contents:
            content = content.replace(" ","")  #去除多余空格
            content_list.append(content[3:])
        time.sleep(15)
        driver.quit()
        print(key_words)
        print(content_list)
    return key_words,content_list


# 将获取的数据存入数据库，这里使用的数据库是mysql
def save_into_database():
    key_words,content_list = get_info()
    #连接mysql数据库
    db = pymysql.connect(host='localhost',user='root',password='199811.',database='entry')#参数依次为IP地址、用户名、密码、具体库
    cursor = db.cursor() #创建游标
    #创建表格，字段分别表示 id、词条、摘要
    sql= """
        create table weapon_info
        (
         id int primary key auto_increment,
         key_words varchar(255),
         contents varchar(1000)
        )charset = utf8;
    """
    #执行sql命令
    cursor.execute(sql)
    #将获取的数据存入数据库
    for i in range(len(key_words)):
        sql1= f"insert into weapon_info set key_words = '{key_words[i]}',contents = '{content_list[i]}'"
        # 执行sql命令
        cursor.execute(sql1)
        db.commit()
        time.sleep(10)
save_into_database()

## 也可将数据作为json格式保存起来
# with open("武器信息.json", 'w', encoding='utf-8') as f:
#     for i in range(len(key_words)):
#         dict = {}
#         # {'词条':'摘要'}
#         dict[key_words[i]] = contents[i]
#         json_data = json.dumps(dict, ensure_ascii=False) + ',\n'
#         f.write(json_data)
# time.sleep(3)